ssh-keygen -t rsa -b 4096

cat ~/.ssh/mykey.pub

ssh -l kemalkova 51.250.72.10 -i ~/.ssh/mykey

Host chip
   Hostname 51.250.72.10
   User kemalkova
   IdentityFile ~/.ssh/mykey

ssh chip

sudo apt update
    sudo apt install nodejs npm

sudo npm install -g @jbrowse/cli

sudo jbrowse create /mnt/ChIP-ATAC-pipe

sudo apt update
    sudo apt install nginx

sudo nano /etc/nginx/nginx.conf

http {
  ...
  include /etc/nginx/conf.d/*.conf;
  #include /etc/nginx/sites-enabled/*;

  server {
          listen 80;
          index index.html;
          server_name 51.250.72.10;

          location /jbrowse/ {
                  alias /mnt/ChIP-ATAC-pipe/;
          }
  }
  ...
}

sudo systemctl restart nginx

sudo apt update
sudo apt install samtools tabix gzip

sudo chown -R $(whoami):$(whoami) /mnt/ChIP-ATAC-pipe/

#!/bin/bash

data_folder=${1:-data}
prefix=${2:-reference_hg38}

mkdir -p "$data_folder"

# Download files
echo "Downloading reference human genome to the $data_folder directory..."

wget -O "$data_folder/$prefix.fa.gz" "https://ftp.ensembl.org/pub/release-108/fasta/homo_sapiens/dna/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz"
wget -O "$data_folder/$prefix.gff3.gz" "https://ftp.ensembl.org/pub/release-108/gff3/homo_sapiens/Homo_sapiens.GRCh38.108.gff3.gz"

echo "Contents of $data_folder after downloading: $(ls $data_folder)"

# Unpack files and fix some gff3 bags
gunzip "$data_folder/$prefix.fa.gz"
zcat "$data_folder/$prefix.gff3.gz" | grep -v "^#" | sort -t$'\t' -k1,1 -k4,4n | bgzip > "$data_folder/$prefix.sorted.gff3.gz"

# Index files
echo "Indexing the fasta file..."
samtools faidx "$data_folder/$prefix.fa"

echo "Indexing the gff3 file..."
tabix -p gff "$data_folder/$prefix.sorted.gff3.gz"

echo "Contents of $data_folder after processing: $(ls $data_folder)"

#!/bin/bash

data_folder=${1:-data}
mkdir -p "$data_folder"

encode_ids=(
  "ENCFF855PCP"
  "ENCFF986SXD"
  "ENCFF708JRU"
  "ENCFF572YTS"
)

# Download files
echo "Downloading ATAC-seq and ChIP-seq data..."
for id in "${encode_ids[@]}"; do
  wget -P "$data_folder" "https://www.encodeproject.org/files/$id/@@download/$id.bed.gz" -q
done

echo "Contents of $data_folder adter downloading:  $(ls $data_folder)"

# Sort, compress and index
for file in "$data_folder"/*.bed.gz; do
  echo "Sorting $file..."
  sorted_file="${file%.bed.gz}.sorted.bed.gz"
  zcat "$file" | sed 's/^chr//' | sort -k1,1 -k2,2n -k6,6 | bgzip > "$sorted_file"

  echo "Indexing $sorted_file..."
  tabix -p bed "$sorted_file"

  echo "Remove initial .bed"
  rm "$file"
done

echo "Contents of $data_folder after .bed processing:  $(ls -lh $data_folder)"

#!/bin/bash

data_folder=${1:-data}
reference_prefix=${2:-reference_hg38}
encode_ids=(
  "ENCFF855PCP"
  "ENCFF986SXD"
  "ENCFF708JRU"
  "ENCFF572YTS"
)

echo "Upload reference genome"
jbrowse add-assembly $data_folder/$reference_prefix.fa --load symlink --force

echo "Upload annotation"
jbrowse add-track $data_folder/$reference_prefix.sorted.gff3.gz --trackType "FeatureTrack" --name "Gene Annotations" --load symlink --force
jbrowse text-index

echo "Upload ChIP- & ATAC-seq data"
for encode_id in "${encode_ids[@]}"; do
  echo "Adding $encode_id..."
  jbrowse add-track "$data_folder/$encode_id.sorted.bed.gz" --name "$encode_id" --load symlink --force
done

SERVER_URL = 'http://51.250.72.10/jbrowse/'
REF_PREFIX = 'reference_hg38'
CHIP_ATAC_IDS = ('ENCFF572YTS', 'ENCFF708JRU', 'ENCFF986SXD', 'ENCFF855PCP')  # the last is ATAC

print('Reference and annotation files:')
for suffix in ('.fa', '.fa.fai', '.sorted.gff3.gz', '.sorted.gff3.gz.tbi'):
    print(f'{SERVER_URL}{REF_PREFIX}{suffix}')

print('\n ChIP- and ATAC-seq files:')

for encode_id in CHIP_ATAC_IDS:
    print(f'{SERVER_URL}{encode_id}.sorted.bed.gz')
    print(f'{SERVER_URL}{encode_id}.sorted.bed.gz.tbi')

Reference and annotation files:
http://51.250.72.10/jbrowse/reference_hg38.fa
http://51.250.72.10/jbrowse/reference_hg38.fa.fai
http://51.250.72.10/jbrowse/reference_hg38.sorted.gff3.gz
http://51.250.72.10/jbrowse/reference_hg38.sorted.gff3.gz.tbi

 ChIP- and ATAC-seq files:
http://51.250.72.10/jbrowse/ENCFF572YTS.sorted.bed.gz
http://51.250.72.10/jbrowse/ENCFF572YTS.sorted.bed.gz.tbi
http://51.250.72.10/jbrowse/ENCFF708JRU.sorted.bed.gz
http://51.250.72.10/jbrowse/ENCFF708JRU.sorted.bed.gz.tbi
http://51.250.72.10/jbrowse/ENCFF986SXD.sorted.bed.gz
http://51.250.72.10/jbrowse/ENCFF986SXD.sorted.bed.gz.tbi
http://51.250.72.10/jbrowse/ENCFF855PCP.sorted.bed.gz
http://51.250.72.10/jbrowse/ENCFF855PCP.sorted.bed.gz.tbi

wget https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-x86_64.sh
    bash Miniconda3-py310_22.11.1-1-Linux-x86_64.sh
    source ~/.bashrc

conda config --add channels conda-forge

# create env
    conda create -c conda-forge -c bioconda -n snakemake snakemake
    # activate env
    conda activate snakemake

rm *.gz *.tbi *.fa *.fa.fai
    rm -r data/

# Snakefile

DATA_FOLDER = "data"
REFERENCE_PREFIX = "reference_hg38"
ENCODE_IDS = ["ENCFF855PCP", "ENCFF986SXD", "ENCFF708JRU", "ENCFF572YTS"]

rule download_reference:
    output:
        fa = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.fa",
        gff3 = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.sorted.gff3.gz"
    shell:
        """
        mkdir -p {DATA_FOLDER}
        wget --no-clobber -O {DATA_FOLDER}/{REFERENCE_PREFIX}.gff3.gz "https://ftp.ensembl.org/pub/release-108/gff3/homo_sapiens/Ho>
        zcat {DATA_FOLDER}/{REFERENCE_PREFIX}.gff3.gz | grep -v "^#" | sort -t$'\t' -k1,1 -k4,4n | bgzip > {output.gff3}
        wget --no-clobber -O {DATA_FOLDER}/{REFERENCE_PREFIX}.fa.gz "https://ftp.ensembl.org/pub/release-108/fasta/homo_sapiens/dna>
        gunzip {DATA_FOLDER}/{REFERENCE_PREFIX}.fa.gz
        """

rule index_files:
    input:
        fa = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.fa",
        gff3 = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.sorted.gff3.gz"
    output:
        fafai = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.fa.fai",
        gff3tbi = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.sorted.gff3.gz.tbi"
    shell:
        """
        samtools faidx {input.fa}
        tabix -p gff {input.gff3}
        """

rule process_chip_seq_data:
    input:
        fa = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.fa",
        gff3 = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.sorted.gff3.gz"
    output:
        sorted_beds = [f"{DATA_FOLDER}/{encode_id}.sorted.bed.gz" for encode_id in ENCODE_IDS]
    shell:
        """
        for encode_id in {ENCODE_IDS}; do
          wget --no-clobber -P {DATA_FOLDER} "https://www.encodeproject.org/files/$encode_id/@@download/$encode_id.bed.gz" -q
          zcat {DATA_FOLDER}/$encode_id.bed.gz | sed 's/^chr//' | sort -k1,1 -k2,2n -k6,6 | bgzip > {DATA_FOLDER}/$encode_id.sorted>
          tabix -p bed {DATA_FOLDER}/$encode_id.sorted.bed.gz
          rm {DATA_FOLDER}/$encode_id.bed.gz
        done
        """

rule upload_to_jbrowse:
    input:
        fa = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.fa",
        gff3 = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.sorted.gff3.gz",
        sorted_beds = [f"{DATA_FOLDER}/{encode_id}.sorted.bed.gz" for encode_id in ENCODE_IDS],
        fafai = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.fa.fai",
        gff3tbi = f"{DATA_FOLDER}/{REFERENCE_PREFIX}.sorted.gff3.gz.tbi"
    output:
        ffa_ = f"{REFERENCE_PREFIX}.fa",                                                                               
        fafai_ = f"{REFERENCE_PREFIX}.fa.fai",
        gff3_ = f"{REFERENCE_PREFIX}.sorted.gff3.gz",
        gff3tbi_ = f"{REFERENCE_PREFIX}.sorted.gff3.gz.tbi",
        sorted_beds_ = [f"{encode_id}.sorted.bed.gz" for encode_id in ENCODE_IDS],
        sorted_beds_tbi = [f"{encode_id}.sorted.bed.gz.tbi" for encode_id in ENCODE_IDS]
    shell:
        """
        jbrowse add-assembly {input.fa} --load symlink --force
        jbrowse add-track {input.gff3} --trackType "FeatureTrack" --name "Gene Annotations" --load symlink --force
        jbrowse text-index
        for file in {input.sorted_beds}; do
           encode_id=$(basename $file)
           jbrowse add-track $file --name $encode_id --load symlink --force
        done
        """

all_target_files = expand("{encode_id}.sorted.bed.gz", encode_id=ENCODE_IDS)
all_target_index_files = expand("{encode_id}.sorted.bed.gz.tbi", encode_id=ENCODE_IDS)

rule all:
    input:
        expand("{REFERENCE_PREFIX}.fa", REFERENCE_PREFIX=REFERENCE_PREFIX),
        expand("{REFERENCE_PREFIX}.fa.fai", REFERENCE_PREFIX=REFERENCE_PREFIX),
        expand("{REFERENCE_PREFIX}.sorted.gff3.gz", REFERENCE_PREFIX=REFERENCE_PREFIX),
        expand("{REFERENCE_PREFIX}.sorted.gff3.gz.tbi", REFERENCE_PREFIX=REFERENCE_PREFIX),
        all_target_files,
        all_target_index_files

snakemake --cores all

ffmpeg -i jbrowse-snakemake.mp4 -r 5 -vf scale=1024:-1 jbrowse-snakemake.gif

Преимущества	Недостатки
Легко настроить и использовать для стандартных задач	Требует изучения синтаксиса (но мне кажется, что если пишешь скрипты на bash, то перейти очень даже легко)
Встроенная поддержка параллелизма и распределенных вычислений	Может быть ограничен в гибкости для более сложных задач
Автоматическое управление зависимостями	Меньше подходит для использования в облаке и на кластерах по сравнению с Nextflow

Преимущества	Недостатки
Легко масштабируется на кластерах и в облаке	Требует более сложного синтаксиса, чем Snakemake
Поддержка контейнеров	Нужно время на изучение, особенно для новичков
Высокая гибкость и возможность адаптации для сложных сценариев	Может быть избыточным для небольших задач

🎓 Homework #2. Snakemake 🎓¶

👤 Assigner:¶

✍️ Contacts:¶

🔥 Deadline 🔥:¶

Theory [2]

Problem [6.5]

Step 1. Start virtual machine [1.5]

Step 2. Start JBrowse repository [1]

Step 3. nginx [1]

Step 4. Load Reference human genome [1]

Step 5. Load ChIP- and ATAC-seq [1]

Step 6. Upload to Genome browser [1]

Extra points [1.5]

Snakemake pipeline [1.25]

Main part [0.25]